{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/kaggle/input/fake-news-machine-hack/Test.csv\n",
      "/kaggle/input/fake-news-machine-hack/Train.csv\n",
      "/kaggle/input/fake-news-machine-hack/sample submission.csv\n"
     ]
    }
   ],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load\n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the read-only \"../input/\" directory\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
    "\n",
    "import os\n",
    "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirname, filename))\n",
    "\n",
    "# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n",
    "# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv(\"/kaggle/input/fake-news-machine-hack/Train.csv\")\n",
    "test = pd.read_csv(\"/kaggle/input/fake-news-machine-hack/Test.csv\")\n",
    "samp = pd.read_csv(\"/kaggle/input/fake-news-machine-hack/sample submission.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Labels</th>\n",
       "      <th>Text</th>\n",
       "      <th>Text_Tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Says the Annies List political group supports ...</td>\n",
       "      <td>abortion</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>When did the decline of coal start? It started...</td>\n",
       "      <td>energy,history,job-accomplishments</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Hillary Clinton agrees with John McCain \"by vo...</td>\n",
       "      <td>foreign-policy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>Health care reform legislation is likely to ma...</td>\n",
       "      <td>health-care</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>The economic turnaround started at the end of ...</td>\n",
       "      <td>economy,jobs</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Labels                                               Text  \\\n",
       "0       1  Says the Annies List political group supports ...   \n",
       "1       2  When did the decline of coal start? It started...   \n",
       "2       3  Hillary Clinton agrees with John McCain \"by vo...   \n",
       "3       1  Health care reform legislation is likely to ma...   \n",
       "4       2  The economic turnaround started at the end of ...   \n",
       "\n",
       "                             Text_Tag  \n",
       "0                            abortion  \n",
       "1  energy,history,job-accomplishments  \n",
       "2                      foreign-policy  \n",
       "3                         health-care  \n",
       "4                        economy,jobs  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "health-care                                      381\n",
       "taxes                                            308\n",
       "immigration                                      253\n",
       "elections                                        252\n",
       "education                                        237\n",
       "                                                ... \n",
       "education,state-budget,state-finances,taxes        1\n",
       "economy,job-accomplishments,message-machine        1\n",
       "energy,environment,public-safety                   1\n",
       "government-regulation,marijuana,public-health      1\n",
       "children,education,legal-issues,sexuality          1\n",
       "Name: Text_Tag, Length: 3827, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['Text_Tag'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2    2114\n",
       "1    1995\n",
       "3    1962\n",
       "5    1676\n",
       "0    1654\n",
       "4     839\n",
       "Name: Labels, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['Labels'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text_Tag                                        Labels\n",
       "Alcohol                                         5         2\n",
       "                                                1         1\n",
       "                                                3         1\n",
       "Alcohol,animals,children,crime                  5         1\n",
       "Alcohol,campaign-finance,ethics,public-service  3         1\n",
       "                                                         ..\n",
       "workers                                         3         3\n",
       "                                                1         2\n",
       "                                                4         2\n",
       "                                                0         1\n",
       "                                                5         1\n",
       "Name: Labels, Length: 5474, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.groupby(['Text_Tag'])['Labels'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Labels  Text_Tag            \n",
       "0       health-care             66\n",
       "        taxes                   50\n",
       "        immigration             42\n",
       "        candidates-biography    39\n",
       "        education               34\n",
       "                                ..\n",
       "5       urban                    1\n",
       "        water                    1\n",
       "        wealth                   1\n",
       "        welfare                  1\n",
       "        workers                  1\n",
       "Name: Text_Tag, Length: 5474, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.groupby(['Labels'])['Text_Tag'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "health-care                                           40\n",
       "elections                                             31\n",
       "education                                             30\n",
       "candidates-biography                                  30\n",
       "economy                                               21\n",
       "                                                      ..\n",
       "candidates-biography,crime,criminal-justice,ethics     1\n",
       "candidates-biography,legal-issues                      1\n",
       "homeland-security,human-rights,immigration             1\n",
       "cap-and-trade,climate-change,environment               1\n",
       "city-budget,labor                                      1\n",
       "Name: Text_Tag, Length: 732, dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test['Text_Tag'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting simpletransformers\n",
      "  Downloading simpletransformers-0.48.1-py3-none-any.whl (209 kB)\n",
      "\u001b[K     |████████████████████████████████| 209 kB 403 kB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: scipy in /opt/conda/lib/python3.7/site-packages (from simpletransformers) (1.4.1)\n",
      "Requirement already satisfied: tensorboardx in /opt/conda/lib/python3.7/site-packages (from simpletransformers) (2.1)\n",
      "Requirement already satisfied: wandb in /opt/conda/lib/python3.7/site-packages (from simpletransformers) (0.9.6)\n",
      "Collecting seqeval\n",
      "  Downloading seqeval-0.0.12.tar.gz (21 kB)\n",
      "Requirement already satisfied: tokenizers in /opt/conda/lib/python3.7/site-packages (from simpletransformers) (0.7.0)\n",
      "Requirement already satisfied: requests in /opt/conda/lib/python3.7/site-packages (from simpletransformers) (2.23.0)\n",
      "Requirement already satisfied: regex in /opt/conda/lib/python3.7/site-packages (from simpletransformers) (2020.4.4)\n",
      "Requirement already satisfied: scikit-learn in /opt/conda/lib/python3.7/site-packages (from simpletransformers) (0.23.2)\n",
      "Requirement already satisfied: pandas in /opt/conda/lib/python3.7/site-packages (from simpletransformers) (1.1.1)\n",
      "Collecting streamlit\n",
      "  Downloading streamlit-0.66.0-py2.py3-none-any.whl (7.2 MB)\n",
      "\u001b[K     |████████████████████████████████| 7.2 MB 4.3 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting transformers>=3.0.2\n",
      "  Downloading transformers-3.1.0-py3-none-any.whl (884 kB)\n",
      "\u001b[K     |████████████████████████████████| 884 kB 8.1 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting tqdm>=4.47.0\n",
      "  Downloading tqdm-4.48.2-py2.py3-none-any.whl (68 kB)\n",
      "\u001b[K     |████████████████████████████████| 68 kB 3.6 MB/s eta 0:00:011\n",
      "\u001b[?25hRequirement already satisfied: numpy in /opt/conda/lib/python3.7/site-packages (from simpletransformers) (1.18.5)\n",
      "Requirement already satisfied: protobuf>=3.8.0 in /opt/conda/lib/python3.7/site-packages (from tensorboardx->simpletransformers) (3.13.0)\n",
      "Requirement already satisfied: six in /opt/conda/lib/python3.7/site-packages (from tensorboardx->simpletransformers) (1.14.0)\n",
      "Requirement already satisfied: subprocess32>=3.5.3 in /opt/conda/lib/python3.7/site-packages (from wandb->simpletransformers) (3.5.4)\n",
      "Requirement already satisfied: python-dateutil>=2.6.1 in /opt/conda/lib/python3.7/site-packages (from wandb->simpletransformers) (2.8.1)\n",
      "Requirement already satisfied: Click>=7.0 in /opt/conda/lib/python3.7/site-packages (from wandb->simpletransformers) (7.1.1)\n",
      "Requirement already satisfied: docker-pycreds>=0.4.0 in /opt/conda/lib/python3.7/site-packages (from wandb->simpletransformers) (0.4.0)\n",
      "Requirement already satisfied: gql==0.2.0 in /opt/conda/lib/python3.7/site-packages (from wandb->simpletransformers) (0.2.0)\n",
      "Requirement already satisfied: PyYAML>=3.10 in /opt/conda/lib/python3.7/site-packages (from wandb->simpletransformers) (5.3.1)\n",
      "Requirement already satisfied: nvidia-ml-py3>=7.352.0 in /opt/conda/lib/python3.7/site-packages (from wandb->simpletransformers) (7.352.0)\n",
      "Requirement already satisfied: sentry-sdk>=0.4.0 in /opt/conda/lib/python3.7/site-packages (from wandb->simpletransformers) (0.17.3)\n",
      "Requirement already satisfied: shortuuid>=0.5.0 in /opt/conda/lib/python3.7/site-packages (from wandb->simpletransformers) (1.0.1)\n",
      "Requirement already satisfied: GitPython>=1.0.0 in /opt/conda/lib/python3.7/site-packages (from wandb->simpletransformers) (3.1.1)\n",
      "Requirement already satisfied: psutil>=5.0.0 in /opt/conda/lib/python3.7/site-packages (from wandb->simpletransformers) (5.7.0)\n",
      "Requirement already satisfied: configparser>=3.8.1 in /opt/conda/lib/python3.7/site-packages (from wandb->simpletransformers) (5.0.0)\n",
      "Requirement already satisfied: watchdog>=0.8.3 in /opt/conda/lib/python3.7/site-packages (from wandb->simpletransformers) (0.10.2)\n",
      "Requirement already satisfied: Keras>=2.2.4 in /opt/conda/lib/python3.7/site-packages (from seqeval->simpletransformers) (2.4.3)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests->simpletransformers) (2020.6.20)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in /opt/conda/lib/python3.7/site-packages (from requests->simpletransformers) (3.0.4)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests->simpletransformers) (2.9)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests->simpletransformers) (1.24.3)\n",
      "Requirement already satisfied: joblib>=0.11 in /opt/conda/lib/python3.7/site-packages (from scikit-learn->simpletransformers) (0.14.1)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /opt/conda/lib/python3.7/site-packages (from scikit-learn->simpletransformers) (2.1.0)\n",
      "Requirement already satisfied: pytz>=2017.2 in /opt/conda/lib/python3.7/site-packages (from pandas->simpletransformers) (2019.3)\n",
      "Collecting astor\n",
      "  Downloading astor-0.8.1-py2.py3-none-any.whl (27 kB)\n",
      "Collecting cachetools>=4.0\n",
      "  Downloading cachetools-4.1.1-py3-none-any.whl (10 kB)\n",
      "Requirement already satisfied: packaging in /opt/conda/lib/python3.7/site-packages (from streamlit->simpletransformers) (20.1)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /opt/conda/lib/python3.7/site-packages (from streamlit->simpletransformers) (7.2.0)\n",
      "Collecting base58\n",
      "  Downloading base58-2.0.1-py3-none-any.whl (4.3 kB)\n",
      "Requirement already satisfied: toml in /opt/conda/lib/python3.7/site-packages (from streamlit->simpletransformers) (0.10.0)\n",
      "Requirement already satisfied: altair>=3.2.0 in /opt/conda/lib/python3.7/site-packages (from streamlit->simpletransformers) (4.1.0)\n",
      "Collecting pydeck>=0.1.dev5\n",
      "  Downloading pydeck-0.5.0b1-py2.py3-none-any.whl (4.4 MB)\n",
      "\u001b[K     |████████████████████████████████| 4.4 MB 8.6 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: tornado>=5.0 in /opt/conda/lib/python3.7/site-packages (from streamlit->simpletransformers) (5.0.2)\n",
      "Requirement already satisfied: blinker in /opt/conda/lib/python3.7/site-packages (from streamlit->simpletransformers) (1.4)\n",
      "Requirement already satisfied: tzlocal in /opt/conda/lib/python3.7/site-packages (from streamlit->simpletransformers) (2.1)\n",
      "Requirement already satisfied: pyarrow in /opt/conda/lib/python3.7/site-packages (from streamlit->simpletransformers) (0.16.0)\n",
      "Collecting enum-compat\n",
      "  Downloading enum_compat-0.0.3-py3-none-any.whl (1.3 kB)\n",
      "Requirement already satisfied: boto3 in /opt/conda/lib/python3.7/site-packages (from streamlit->simpletransformers) (1.14.56)\n",
      "Requirement already satisfied: botocore>=1.13.44 in /opt/conda/lib/python3.7/site-packages (from streamlit->simpletransformers) (1.17.56)\n",
      "Collecting validators\n",
      "  Downloading validators-0.18.1-py3-none-any.whl (19 kB)\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.7/site-packages (from transformers>=3.0.2->simpletransformers) (3.0.10)\n",
      "Requirement already satisfied: sentencepiece!=0.1.92 in /opt/conda/lib/python3.7/site-packages (from transformers>=3.0.2->simpletransformers) (0.1.91)\n",
      "Requirement already satisfied: sacremoses in /opt/conda/lib/python3.7/site-packages (from transformers>=3.0.2->simpletransformers) (0.0.43)\n",
      "Requirement already satisfied: setuptools in /opt/conda/lib/python3.7/site-packages (from protobuf>=3.8.0->tensorboardx->simpletransformers) (46.1.3.post20200325)\n",
      "Requirement already satisfied: promise<3,>=2.0 in /opt/conda/lib/python3.7/site-packages (from gql==0.2.0->wandb->simpletransformers) (2.3)\n",
      "Requirement already satisfied: graphql-core<2,>=0.5.0 in /opt/conda/lib/python3.7/site-packages (from gql==0.2.0->wandb->simpletransformers) (1.1)\n",
      "Requirement already satisfied: gitdb<5,>=4.0.1 in /opt/conda/lib/python3.7/site-packages (from GitPython>=1.0.0->wandb->simpletransformers) (4.0.4)\n",
      "Requirement already satisfied: pathtools>=0.1.1 in /opt/conda/lib/python3.7/site-packages (from watchdog>=0.8.3->wandb->simpletransformers) (0.1.2)\n",
      "Requirement already satisfied: h5py in /opt/conda/lib/python3.7/site-packages (from Keras>=2.2.4->seqeval->simpletransformers) (2.10.0)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /opt/conda/lib/python3.7/site-packages (from packaging->streamlit->simpletransformers) (2.4.7)\n",
      "Requirement already satisfied: entrypoints in /opt/conda/lib/python3.7/site-packages (from altair>=3.2.0->streamlit->simpletransformers) (0.3)\n",
      "Requirement already satisfied: jinja2 in /opt/conda/lib/python3.7/site-packages (from altair>=3.2.0->streamlit->simpletransformers) (2.11.2)\n",
      "Requirement already satisfied: jsonschema in /opt/conda/lib/python3.7/site-packages (from altair>=3.2.0->streamlit->simpletransformers) (3.2.0)\n",
      "Requirement already satisfied: toolz in /opt/conda/lib/python3.7/site-packages (from altair>=3.2.0->streamlit->simpletransformers) (0.10.0)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: traitlets>=4.3.2 in /opt/conda/lib/python3.7/site-packages (from pydeck>=0.1.dev5->streamlit->simpletransformers) (4.3.3)\n",
      "Requirement already satisfied: ipywidgets>=7.0.0 in /opt/conda/lib/python3.7/site-packages (from pydeck>=0.1.dev5->streamlit->simpletransformers) (7.5.1)\n",
      "Collecting ipykernel>=5.1.2; python_version >= \"3.4\"\n",
      "  Downloading ipykernel-5.3.4-py3-none-any.whl (120 kB)\n",
      "\u001b[K     |████████████████████████████████| 120 kB 13.0 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: jmespath<1.0.0,>=0.7.1 in /opt/conda/lib/python3.7/site-packages (from boto3->streamlit->simpletransformers) (0.10.0)\n",
      "Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /opt/conda/lib/python3.7/site-packages (from boto3->streamlit->simpletransformers) (0.3.3)\n",
      "Requirement already satisfied: docutils<0.16,>=0.10 in /opt/conda/lib/python3.7/site-packages (from botocore>=1.13.44->streamlit->simpletransformers) (0.15.2)\n",
      "Requirement already satisfied: decorator>=3.4.0 in /opt/conda/lib/python3.7/site-packages (from validators->streamlit->simpletransformers) (4.4.2)\n",
      "Requirement already satisfied: smmap<4,>=3.0.1 in /opt/conda/lib/python3.7/site-packages (from gitdb<5,>=4.0.1->GitPython>=1.0.0->wandb->simpletransformers) (3.0.2)\n",
      "Requirement already satisfied: MarkupSafe>=0.23 in /opt/conda/lib/python3.7/site-packages (from jinja2->altair>=3.2.0->streamlit->simpletransformers) (1.1.1)\n",
      "Requirement already satisfied: attrs>=17.4.0 in /opt/conda/lib/python3.7/site-packages (from jsonschema->altair>=3.2.0->streamlit->simpletransformers) (19.3.0)\n",
      "Requirement already satisfied: pyrsistent>=0.14.0 in /opt/conda/lib/python3.7/site-packages (from jsonschema->altair>=3.2.0->streamlit->simpletransformers) (0.16.0)\n",
      "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /opt/conda/lib/python3.7/site-packages (from jsonschema->altair>=3.2.0->streamlit->simpletransformers) (1.7.0)\n",
      "Requirement already satisfied: ipython-genutils in /opt/conda/lib/python3.7/site-packages (from traitlets>=4.3.2->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.2.0)\n",
      "Requirement already satisfied: ipython>=4.0.0; python_version >= \"3.3\" in /opt/conda/lib/python3.7/site-packages (from ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (7.13.0)\n",
      "Requirement already satisfied: nbformat>=4.2.0 in /opt/conda/lib/python3.7/site-packages (from ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (5.0.6)\n",
      "Requirement already satisfied: widgetsnbextension~=3.5.0 in /opt/conda/lib/python3.7/site-packages (from ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (3.5.1)\n",
      "Requirement already satisfied: jupyter-client in /opt/conda/lib/python3.7/site-packages (from ipykernel>=5.1.2; python_version >= \"3.4\"->pydeck>=0.1.dev5->streamlit->simpletransformers) (6.1.3)\n",
      "Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata; python_version < \"3.8\"->jsonschema->altair>=3.2.0->streamlit->simpletransformers) (3.1.0)\n",
      "Requirement already satisfied: backcall in /opt/conda/lib/python3.7/site-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.1.0)\n",
      "Requirement already satisfied: pexpect; sys_platform != \"win32\" in /opt/conda/lib/python3.7/site-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (4.8.0)\n",
      "Requirement already satisfied: pygments in /opt/conda/lib/python3.7/site-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (2.6.1)\n",
      "Requirement already satisfied: pickleshare in /opt/conda/lib/python3.7/site-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.7.5)\n",
      "Requirement already satisfied: jedi>=0.10 in /opt/conda/lib/python3.7/site-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.15.2)\n",
      "Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in /opt/conda/lib/python3.7/site-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (3.0.5)\n",
      "Requirement already satisfied: jupyter-core in /opt/conda/lib/python3.7/site-packages (from nbformat>=4.2.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (4.6.3)\n",
      "Requirement already satisfied: notebook>=4.4.1 in /opt/conda/lib/python3.7/site-packages (from widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (5.5.0)\n",
      "Requirement already satisfied: pyzmq>=13 in /opt/conda/lib/python3.7/site-packages (from jupyter-client->ipykernel>=5.1.2; python_version >= \"3.4\"->pydeck>=0.1.dev5->streamlit->simpletransformers) (19.0.0)\n",
      "Requirement already satisfied: ptyprocess>=0.5 in /opt/conda/lib/python3.7/site-packages (from pexpect; sys_platform != \"win32\"->ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.6.0)\n",
      "Requirement already satisfied: parso>=0.5.2 in /opt/conda/lib/python3.7/site-packages (from jedi>=0.10->ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.5.2)\n",
      "Requirement already satisfied: wcwidth in /opt/conda/lib/python3.7/site-packages (from prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0->ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.1.9)\n",
      "Requirement already satisfied: terminado>=0.8.1 in /opt/conda/lib/python3.7/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.8.3)\n",
      "Requirement already satisfied: nbconvert in /opt/conda/lib/python3.7/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (5.6.1)\n",
      "Requirement already satisfied: Send2Trash in /opt/conda/lib/python3.7/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (1.5.0)\n",
      "Requirement already satisfied: bleach in /opt/conda/lib/python3.7/site-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (3.1.4)\n",
      "Requirement already satisfied: defusedxml in /opt/conda/lib/python3.7/site-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.6.0)\n",
      "Requirement already satisfied: mistune<2,>=0.8.1 in /opt/conda/lib/python3.7/site-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.8.4)\n",
      "Requirement already satisfied: testpath in /opt/conda/lib/python3.7/site-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.4.4)\n",
      "Requirement already satisfied: pandocfilters>=1.4.1 in /opt/conda/lib/python3.7/site-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (1.4.2)\n",
      "Requirement already satisfied: webencodings in /opt/conda/lib/python3.7/site-packages (from bleach->nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.5.1)\n",
      "Building wheels for collected packages: seqeval\n",
      "  Building wheel for seqeval (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for seqeval: filename=seqeval-0.0.12-py3-none-any.whl size=7423 sha256=3b4024985a424447742f1510d6c51dcdf5ba0729cf8ea26e06abe5e8ce59fc13\n",
      "  Stored in directory: /root/.cache/pip/wheels/dc/cc/62/a3b81f92d35a80e39eb9b2a9d8b31abac54c02b21b2d466edc\n",
      "Successfully built seqeval\n",
      "Installing collected packages: seqeval, astor, cachetools, base58, ipykernel, pydeck, enum-compat, validators, streamlit, tqdm, transformers, simpletransformers\n",
      "  Attempting uninstall: cachetools\n",
      "    Found existing installation: cachetools 3.1.1\n",
      "    Uninstalling cachetools-3.1.1:\n",
      "      Successfully uninstalled cachetools-3.1.1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Attempting uninstall: ipykernel\n",
      "    Found existing installation: ipykernel 5.1.1\n",
      "    Uninstalling ipykernel-5.1.1:\n",
      "      Successfully uninstalled ipykernel-5.1.1\n",
      "  Attempting uninstall: tqdm\n",
      "    Found existing installation: tqdm 4.45.0\n",
      "    Uninstalling tqdm-4.45.0:\n",
      "      Successfully uninstalled tqdm-4.45.0\n",
      "  Attempting uninstall: transformers\n",
      "    Found existing installation: transformers 2.11.0\n",
      "    Uninstalling transformers-2.11.0:\n",
      "      Successfully uninstalled transformers-2.11.0\n",
      "\u001b[31mERROR: After October 2020 you may experience errors when installing or updating packages. This is because pip will change the way that it resolves dependency conflicts.\n",
      "\n",
      "We recommend you use --use-feature=2020-resolver to test your packages with the new resolver before it becomes the default.\n",
      "\n",
      "transformers 3.1.0 requires tokenizers==0.8.1.rc2, but you'll have tokenizers 0.7.0 which is incompatible.\n",
      "jupyterlab-git 0.10.0 requires nbdime<2.0.0,>=1.1.0, but you'll have nbdime 2.0.0 which is incompatible.\n",
      "allennlp 1.0.0 requires transformers<2.12,>=2.9, but you'll have transformers 3.1.0 which is incompatible.\u001b[0m\n",
      "Successfully installed astor-0.8.1 base58-2.0.1 cachetools-4.1.1 enum-compat-0.0.3 ipykernel-5.3.4 pydeck-0.5.0b1 seqeval-0.0.12 simpletransformers-0.48.1 streamlit-0.66.0 tqdm-4.48.2 transformers-3.1.0 validators-0.18.1\n",
      "\u001b[33mWARNING: You are using pip version 20.2.2; however, version 20.2.3 is available.\n",
      "You should consider upgrading via the '/opt/conda/bin/python3.7 -m pip install --upgrade pip' command.\u001b[0m\n",
      "Found existing installation: tokenizers 0.7.0\n",
      "Uninstalling tokenizers-0.7.0:\n",
      "  Successfully uninstalled tokenizers-0.7.0\n",
      "Requirement already satisfied: transformers in /opt/conda/lib/python3.7/site-packages (3.1.0)\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.7/site-packages (from transformers) (3.0.10)\n",
      "Requirement already satisfied: packaging in /opt/conda/lib/python3.7/site-packages (from transformers) (20.1)\n",
      "Requirement already satisfied: requests in /opt/conda/lib/python3.7/site-packages (from transformers) (2.23.0)\n",
      "Requirement already satisfied: sacremoses in /opt/conda/lib/python3.7/site-packages (from transformers) (0.0.43)\n",
      "Requirement already satisfied: sentencepiece!=0.1.92 in /opt/conda/lib/python3.7/site-packages (from transformers) (0.1.91)\n",
      "Requirement already satisfied: numpy in /opt/conda/lib/python3.7/site-packages (from transformers) (1.18.5)\n",
      "Collecting tokenizers==0.8.1.rc2\n",
      "  Downloading tokenizers-0.8.1rc2-cp37-cp37m-manylinux1_x86_64.whl (3.0 MB)\n",
      "\u001b[K     |████████████████████████████████| 3.0 MB 3.8 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.7/site-packages (from transformers) (4.48.2)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.7/site-packages (from transformers) (2020.4.4)\n",
      "Requirement already satisfied: six in /opt/conda/lib/python3.7/site-packages (from packaging->transformers) (1.14.0)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /opt/conda/lib/python3.7/site-packages (from packaging->transformers) (2.4.7)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests->transformers) (2.9)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests->transformers) (2020.6.20)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in /opt/conda/lib/python3.7/site-packages (from requests->transformers) (3.0.4)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests->transformers) (1.24.3)\n",
      "Requirement already satisfied: click in /opt/conda/lib/python3.7/site-packages (from sacremoses->transformers) (7.1.1)\n",
      "Requirement already satisfied: joblib in /opt/conda/lib/python3.7/site-packages (from sacremoses->transformers) (0.14.1)\n",
      "Installing collected packages: tokenizers\n",
      "\u001b[31mERROR: After October 2020 you may experience errors when installing or updating packages. This is because pip will change the way that it resolves dependency conflicts.\n",
      "\n",
      "We recommend you use --use-feature=2020-resolver to test your packages with the new resolver before it becomes the default.\n",
      "\n",
      "allennlp 1.0.0 requires transformers<2.12,>=2.9, but you'll have transformers 3.1.0 which is incompatible.\u001b[0m\n",
      "Successfully installed tokenizers-0.8.1rc2\n",
      "\u001b[33mWARNING: You are using pip version 20.2.2; however, version 20.2.3 is available.\n",
      "You should consider upgrading via the '/opt/conda/bin/python3.7 -m pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install simpletransformers\n",
    "!pip uninstall -y tokenizers\n",
    "!pip install transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def random_seed(seed_value):\n",
    "    import random \n",
    "    random.seed(seed_value)  \n",
    "    import numpy as np\n",
    "    np.random.seed(seed_value)  \n",
    "    import torch\n",
    "    torch.manual_seed(seed_value)  \n",
    "    \n",
    "    if torch.cuda.is_available(): \n",
    "        torch.cuda.manual_seed(seed_value)\n",
    "        torch.cuda.manual_seed_all(seed_value)  \n",
    "        torch.backends.cudnn.deterministic = True   \n",
    "        torch.backends.cudnn.benchmark = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import gc\n",
    "import re\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "from sklearn.model_selection import train_test_split, StratifiedKFold, KFold\n",
    "from sklearn.metrics import accuracy_score, log_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Labels</th>\n",
       "      <th>Text</th>\n",
       "      <th>Text_Tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1846</th>\n",
       "      <td>1</td>\n",
       "      <td>Obama says Iran is a 'tiny' country, 'doesn't ...</td>\n",
       "      <td>foreign-policy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2697</th>\n",
       "      <td>1</td>\n",
       "      <td>On repealing the 17th Amendment</td>\n",
       "      <td>debates,elections,states</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4386</th>\n",
       "      <td>1</td>\n",
       "      <td>On the Trans-Pacific Partnership.</td>\n",
       "      <td>trade</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4839</th>\n",
       "      <td>2</td>\n",
       "      <td>During Sherrod Browns past decade as a D.C. po...</td>\n",
       "      <td>economy,job-accomplishments,jobs</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4940</th>\n",
       "      <td>1</td>\n",
       "      <td>On changing the rules for filibusters on presi...</td>\n",
       "      <td>congressional-rules</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6784</th>\n",
       "      <td>1</td>\n",
       "      <td>On support for the Export-Import Bank</td>\n",
       "      <td>trade</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8906</th>\n",
       "      <td>5</td>\n",
       "      <td>Says Mitt Romney flip-flopped on abortion.</td>\n",
       "      <td>abortion,message-machine-2012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9642</th>\n",
       "      <td>1</td>\n",
       "      <td>On changing the rules for filibusters on presi...</td>\n",
       "      <td>congressional-rules</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Labels                                               Text  \\\n",
       "1846       1  Obama says Iran is a 'tiny' country, 'doesn't ...   \n",
       "2697       1                    On repealing the 17th Amendment   \n",
       "4386       1                  On the Trans-Pacific Partnership.   \n",
       "4839       2  During Sherrod Browns past decade as a D.C. po...   \n",
       "4940       1  On changing the rules for filibusters on presi...   \n",
       "6784       1              On support for the Export-Import Bank   \n",
       "8906       5         Says Mitt Romney flip-flopped on abortion.   \n",
       "9642       1  On changing the rules for filibusters on presi...   \n",
       "\n",
       "                              Text_Tag  \n",
       "1846                    foreign-policy  \n",
       "2697          debates,elections,states  \n",
       "4386                             trade  \n",
       "4839  economy,job-accomplishments,jobs  \n",
       "4940               congressional-rules  \n",
       "6784                             trade  \n",
       "8906     abortion,message-machine-2012  \n",
       "9642               congressional-rules  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train[train.duplicated()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(train.shape)\n",
    "# train = train.drop_duplicates()\n",
    "# print(train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Text_Tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [Text, Text_Tag]\n",
       "Index: []"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test[test.duplicated()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = train.rename(columns={'Text' : 'text', 'Labels': 'labels'})\n",
    "test = test.rename(columns={'Text' : 'text'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "train['text'] = train['text'] + ' ' +  train['Text_Tag'].fillna(\"\").apply(lambda x : ' '.join(x.split(',')))\n",
    "test['text'] = test['text'] + ' ' +  test['Text_Tag'].fillna(\"\").apply(lambda x : ' '.join(x.split(',')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "train.drop('Text_Tag', axis=1, inplace=True)\n",
    "test.drop('Text_Tag', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count    10240.000000\n",
      "mean        20.178320\n",
      "std          9.832896\n",
      "min          3.000000\n",
      "25%         14.000000\n",
      "50%         19.000000\n",
      "75%         25.000000\n",
      "max        470.000000\n",
      "Name: text, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "print(train['text'].apply(lambda x: len(x.split())).describe())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count    1267.000000\n",
      "mean       20.546172\n",
      "std        14.948253\n",
      "min         3.000000\n",
      "50%        19.000000\n",
      "80%        26.800000\n",
      "90%        31.000000\n",
      "95%        35.000000\n",
      "99%        43.340000\n",
      "max       432.000000\n",
      "Name: text, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "print(test['text'].apply(lambda x: len(x.split())).describe([0.8,0.9,0.95,0.99]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = train[['text','labels']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m W&B installed but not logged in.  Run `wandb login` or set the WANDB_API_KEY env variable.\n"
     ]
    }
   ],
   "source": [
    "from simpletransformers.classification import ClassificationModel\n",
    "from sklearn.model_selection import train_test_split, StratifiedKFold, KFold\n",
    "from sklearn.metrics import accuracy_score, log_loss\n",
    "from scipy.special import softmax\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_args = {'train_batch_size': 32, \n",
    "              'reprocess_input_data': True,\n",
    "              'overwrite_output_dir': True,\n",
    "              'fp16': False,\n",
    "              'do_lower_case': False,\n",
    "              'num_train_epochs': 4,\n",
    "              'max_seq_length': 44,\n",
    "              'regression': False,\n",
    "              'manual_seed': 1994,\n",
    "              \"learning_rate\": 1e-5,\n",
    "              #'weight_decay': 0.01,\n",
    "              \"save_eval_checkpoints\": False,\n",
    "              \"save_model_every_epoch\": False,\n",
    "              'no_cache':True,\n",
    "              \"silent\": True,\n",
    "              \"use_early_stopping\": True,\n",
    "              #\"early_stopping_delta\": 0.015,\n",
    "              \"early_stopping_metric\": \"mcc\",\n",
    "              \"early_stopping_metric_minimize\": False,\n",
    "              \"early_stopping_patience\": 3,\n",
    "              \"evaluate_during_training_steps\": 1000\n",
    "              }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_model():\n",
    "    model = ClassificationModel('roberta', 'roberta-large', use_cuda=True, num_labels=6, args=model_args)                            \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "err=[]\n",
    "y_pred_tot=[]\n",
    "i=1\n",
    "\n",
    "fold=StratifiedKFold(n_splits=10, shuffle=True, random_state=1994)\n",
    "\n",
    "for train_index, test_index in fold.split(train, train['labels']):\n",
    "    train1_trn, train1_val = train.iloc[train_index], train.iloc[test_index]\n",
    "    model = get_model()\n",
    "    gc.collect()\n",
    "\n",
    "    model.train_model(train1_trn)\n",
    "    score, raw_outputs_val, wrong_preds = model.eval_model(train1_val) \n",
    "    raw_outputs_val = softmax(raw_outputs_val, axis=1)\n",
    "    #raw_outputs_val = np.clip(raw_outputs_val,0.05,0.95)\n",
    "    print('Log_Loss:', log_loss(train1_val['labels'], raw_outputs_val))\n",
    "    err.append(log_loss(train1_val['labels'], raw_outputs_val))\n",
    "    predictions, raw_outputs_test = model.predict(test['text'])\n",
    "    raw_outputs_test = softmax(raw_outputs_test, axis=1) \n",
    "    y_pred_tot.append(raw_outputs_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.mean(err, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1.6813"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = np.mean(y_pred_tot, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "samp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "samp[['0','1','2','3','4','5']] = y_pred\n",
    "samp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv(\"/kaggle/input/fake-news-machine-hack/Train.csv\")\n",
    "test = pd.read_csv(\"/kaggle/input/fake-news-machine-hack/Test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_ids = []\n",
    "force_senti = []\n",
    "for index,row in test.iterrows():\n",
    "    temp = train[train['Text'] == row['Text']]\n",
    "    #temp = temp[temp['Text_Tag'] == row['Text_Tag']]\n",
    "    if temp.shape[0] > 0:\n",
    "        print(temp.shape)\n",
    "        force_senti.append(list(set(temp['Labels'].tolist()))[0])\n",
    "        test_ids.append(index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dic = {0:[1,0,0,0,0,0],1:[0,1,0,0,0,0],2:[0,0,1,0,0,0],3:[0,0,0,1,0,0],4:[0,0,0,0,1,0],5:[0,0,0,0,0,1]}\n",
    "for x,y in zip(test_ids,force_senti):\n",
    "    #print(samp.iloc[x])\n",
    "    target = dic[y]\n",
    "    samp.iloc[x] = target\n",
    "    #print(samp.iloc[x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "samp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "samp.to_csv('Sub_v1.0.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
